Merge pull request #1079 from cantino/rss_agent_sorts_all_events
Gather all Events in RSSAgent before sorting
Andrew Cantino
1 changed files with 26 additions and 19 deletions
+ 26
- 19
app/models/agents/rss_agent.rb
@@ -87,34 +87,41 @@ module Agents |
||
| 87 | 87 |
end |
| 88 | 88 |
|
| 89 | 89 |
def check |
| 90 |
- Array(interpolated['url']).each do |url| |
|
| 91 |
- check_url(url) |
|
| 92 |
- end |
|
| 90 |
+ check_urls(Array(interpolated['url'])) |
|
| 93 | 91 |
end |
| 94 | 92 |
|
| 95 | 93 |
protected |
| 96 | 94 |
|
| 97 |
- def check_url(url) |
|
| 98 |
- response = faraday.get(url) |
|
| 99 |
- if response.success? |
|
| 100 |
- feed = FeedNormalizer::FeedNormalizer.parse(response.body, loose: true) |
|
| 101 |
- feed.clean! if boolify(interpolated['clean']) |
|
| 102 |
- max_events = (interpolated['max_events_per_run'].presence || 0).to_i |
|
| 103 |
- created_event_count = 0 |
|
| 104 |
- sort_events(feed_to_events(feed)).each.with_index do |event, index| |
|
| 105 |
- break if max_events && max_events > 0 && index >= max_events |
|
| 106 |
- entry_id = event.payload[:id] |
|
| 107 |
- if check_and_track(entry_id) |
|
| 95 |
+ def check_urls(urls) |
|
| 96 |
+ new_events = [] |
|
| 97 |
+ max_events = (interpolated['max_events_per_run'].presence || 0).to_i |
|
| 98 |
+ |
|
| 99 |
+ urls.each do |url| |
|
| 100 |
+ begin |
|
| 101 |
+ response = faraday.get(url) |
|
| 102 |
+ if response.success? |
|
| 103 |
+ feed = FeedNormalizer::FeedNormalizer.parse(response.body, loose: true) |
|
| 104 |
+ feed.clean! if boolify(interpolated['clean']) |
|
| 105 |
+ new_events.concat feed_to_events(feed) |
|
| 106 |
+ else |
|
| 107 |
+ error "Failed to fetch #{url}: #{response.inspect}"
|
|
| 108 |
+ end |
|
| 109 |
+ rescue => e |
|
| 110 |
+ error "Failed to fetch #{url} with message '#{e.message}': #{e.backtrace}"
|
|
| 111 |
+ end |
|
| 112 |
+ end |
|
| 113 |
+ |
|
| 114 |
+ created_event_count = 0 |
|
| 115 |
+ sort_events(new_events).each.with_index do |event, index| |
|
| 116 |
+ entry_id = event.payload[:id] |
|
| 117 |
+ if check_and_track(entry_id) |
|
| 118 |
+ unless max_events && max_events > 0 && index >= max_events |
|
| 108 | 119 |
created_event_count += 1 |
| 109 | 120 |
create_event(event) |
| 110 | 121 |
end |
| 111 | 122 |
end |
| 112 |
- log "Fetched #{url} and created #{created_event_count} event(s)."
|
|
| 113 |
- else |
|
| 114 |
- error "Failed to fetch #{url}: #{response.inspect}"
|
|
| 115 | 123 |
end |
| 116 |
- rescue => e |
|
| 117 |
- error "Failed to fetch #{url} with message '#{e.message}': #{e.backtrace}"
|
|
| 124 |
+ log "Fetched #{urls.to_sentence} and created #{created_event_count} event(s)."
|
|
| 118 | 125 |
end |
| 119 | 126 |
|
| 120 | 127 |
def get_entry_id(entry) |